#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Get proxies from urls, and test their speed"""
import urllib, re, time, threading


urls = ["http://proxy.ipcn.org/proxya2.html",
        "http://www.cz88.net/proxy/index.shtml",
        "http://www.cz88.net/proxy/http_2.shtml",
        "http://www.cz88.net/proxy/http_3.shtml",
         "http://www.cz88.net/proxy/http_4.shtml",
         "http://www.cz88.net/proxy/http_5.shtml",
         "http://www.cz88.net/proxy/http_6.shtml",
         "http://www.cz88.net/proxy/http_7.shtml",
         "http://www.cz88.net/proxy/http_8.shtml",
         "http://www.cz88.net/proxy/http_9.shtml",
         "http://www.cz88.net/proxy/http_10.shtml",
#        "http://proxy.ipcn.org/proxylist.html",
 #       "http://proxy.ipcn.org/proxylist2.html",
  #      "http://www.xjtushare.com/proxy.aspx", 
   #     "http://info.hustonline.net/index/proxyshow.aspx",
    #    "http://youth.bupt.edu.cn/proxy/txt.jsp"
        ]		#where to get proxies
urls_proxy = {}		#proxy used to connect urls
proxy_pattern = [r"""(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d{1,})""",
                 r"""<tr><td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td><td>(\d{1,})</td>""",]

test_url = "http://video.google.com/"
dnslook_url = "http://www.cz88.net/ip/ipcheck.aspx?ip="
test_pattern = re.compile(r"""Upload Your Videos""")
time_out = 15.0		#max waiting time to test proxies
output_file = "Proxies.txt"


class TestTime(threading.Thread):
	"""test a proxy's speed in new thread by recording its connect time"""
	def __init__(self, proxy):
		threading.Thread.__init__(self)
		self.proxy = proxy
		self.time = None
		self.stat = proxy + " time out!"
        def __del__(self):
                self = None
	def run(self):
		start = time.time()
		try:
			f = urllib.urlopen(test_url, proxies = {"http":"http://"+self.proxy})
		except:
			self.stat = self.proxy+" fails!"
		else:
			data = f.read()
			f.close()
			end = time.time()
			if test_pattern.search(data): #if data is matched
				self.time = end-start
				self.stat = self.proxy+" time: "+str(self.time)
			else:
				self.stat = self.proxy+" not matched!"
				

def getlocation(ip):
        if ':' in ip:
                ip = ip[:ip.index(':')]
        try:
                print "get location of "+ip +" from: "+dnslook_url+ip
                f = urllib.urlopen(dnslook_url+ip)
        except:
                print "error , timeout"
                return "Location unknown"
        else:
                data = f.read()
                f.close()
                loc = re.findall(r'document.write\("(.*)"\)', data)
                if len(loc)>0:
                        return "Location "+loc[0]
                else:
                        return "Location unknown"
def totest(proxy, result):
	"""test a proxy's speed in time_out seconds"""
	test = TestTime(proxy)
	test.setDaemon(False) 
	print "testing "+proxy
	test.start()
        try:
                test.join(time_out) 	#wait time_out seconds for testing
                print test.stat
                if test.time:
                        result.append((test.time, proxy))
        except:
                test = None
			
def fetchproxylist(set_ofproxies, data):
        for patstr in proxy_pattern:
                set_ofproxies.update([ippart+":"+ipport for (ippart, ipport) in re.findall(patstr, data)])
        
if __name__ == "__main__":	
	#get old proxies in output_file
	try:
		f = open(output_file)
	except:
		allproxies = set()
	else:
		allproxies = set([x[:-1] for x in f.readlines()])
		f.close()	

        result = []	
	#get else proxies from urls
	for url in urls:
                allproxies.clear()
		print "getting proxy from "+url
		try:
			f = urllib.urlopen(url, proxies=urls_proxy)
		except:
			print url+" can not open!\n"
		else:
			data = f.read()
			f.close()
                        fetchproxylist(allproxies, data)
			print url+" finished!"

                        #test all proxies' speed
                        for proxy in allproxies:
                                #new thread to test every proxy
                                t = threading.Thread(target=totest, args=(proxy, result))
                                t.setDaemon(True)
                                t.start()
                        time.sleep(time_out+5.0)
	#show all proxies' speed
	result.sort()
	for i in xrange(len(result)):
		print str(i+1)+"\t"+result[i][1]+"   \t:\t"+str(result[i][0])
	
	#output needed proxies	
	num = min(abs(int(raw_input("\nHow many proxies to output: "))), len(result))
	try:
		f = open(output_file, "w")
	except:
		print "Can not open output file!"
	else:
		f.writelines([x[1]+" "+getlocation(x[1])+"\n" for x in result[:num]])
		f.close()
		print str(num)+" proxies are output."
	
